#!/usr/bin/env python
#coding=utf-8
import urllib,urllib2
import re
import StringIO
import gzip

from HTMLParser import HTMLParser

## my python file.
#import dbcon


class Rhea_Amazon_html_manager():
    rhea_Amazon_main_page = ''
    rhea_Amazon_phone_page = ''
    rhea_Amazon_phone_page0 = ''
    
    rhea_Amazon_current_page = 0
    rhea_Amazon_has_next_page = True



class Rhea_360buy_html_manager():
    
    ## page phone main page
    rhea_360buy_main_page = "http://www.360buy.com/products/652-653-655.html"
    
    ## page list data
    rhea_360buy_page_lists = []
    rhea_360buy_page_num = 0
    rhea_360buy_page_pattern = r"652-653-655-0-0-0-0-0-0-0-1-1-\d{1,}.html"
    rhea_360buy_page_head =  "652-653-655-0-0-0-0-0-0-0-1-1-"
    rhea_360buy_page_foot = ".html"

class Rhea_360buy_Phone_Parser(HTMLParser):
    
    
    rhea_tag_div = 'div'
    rhea_tag_href = 'href'
    
    rhea_list_phone_tag = 'ul'
    rhea_list_phone_class = ('class','list-h')
    
    rhea_360_phone_img_class = ('class','p-img')
    rhea_360_phone_name_class = ('class','p-name')
    rhea_360_phone_price_class = ('class','p-price')

    
    rhea_tag_find = False
    rhea_tag_phone_find = False
    rhea_tag_img_find = False
    rhea_tag_name_find = False
    rhea_tag_price_find = False
    
    rhea_360_phone_id =''
    rhea_360_phone_img =''
    rhea_360_phone_name=''
    rhea_360_phone_url=''
    rhea_360_phone_price=''
    rhea_360_phone_hot=''
    
    
    
    ##db connection
    rhea_db_handle = 1

    
    def handle_starttag(self, tag, attrs):
        if self.rhea_list_phone_tag == tag and self.rhea_list_phone_class in attrs:
            self.rhea_tag_find = True
            #print "Encountered a start tag:", tag, attrs
            
        if  self.rhea_tag_find:
            #print "Encountered a start tag1:", tag, attrs  
            
            if 'li' == tag and 'sku' in attrs[0][0]:
                self.rhea_tag_phone_find = True
                self.rhea_tag_img_find = False
                self.rhea_tag_name_find  = False
                self.rhea_tag_price_find  = False
                self.rhea_360_phone_id = attrs[0][1]

            if tag == self.rhea_tag_div and self.rhea_360_phone_img_class in attrs:
                #print "find img tag"
                self.rhea_tag_img_find  = True

                
            
            #img
            if  self.rhea_tag_img_find:
                if tag == 'a' and self.rhea_tag_href in attrs[1]:
                    self.rhea_360_phone_url = attrs[1][1]
                if tag == 'img':
                    self.rhea_360_phone_img = attrs[1][1]
                    self.rhea_360_phone_hot = attrs[4][1]
            
            #name    
            if tag == self.rhea_tag_div and self.rhea_360_phone_name_class in attrs:
                self.rhea_tag_name_find = True
            
            # price
            if tag == self.rhea_tag_div and self.rhea_360_phone_price_class in attrs:
                self.rhea_tag_price_find = True            
            if self.rhea_tag_price_find:
                if tag == 'img' and 'src' in attrs[1]:
                    self.rhea_360_phone_price = attrs[1][1]     
                

    
    def handle_endtag(self, tag):
        if  self.rhea_tag_find:
            #print "Encountered an end tag :", tag
            if self.rhea_list_phone_tag == tag:
                self.rhea_tag_find = False
            
            if self.rhea_tag_find:
                if  tag == self.rhea_tag_div and self.rhea_tag_img_find:
                    self.rhea_tag_img_find = False
                if  tag == self.rhea_tag_div and self.rhea_tag_name_find:
                    self.rhea_tag_name_find = False    
                if  tag == self.rhea_tag_div and self.rhea_tag_price_find:
                    self.rhea_tag_price_find = False   
                    self.rhea_tag_phone_find = False
                    print "---------------"
                    print self.rhea_360_phone_id
                    print self.rhea_360_phone_url
                    print self.rhea_360_phone_img
                    print self.rhea_360_phone_price
                    #print self.rhea_360_phone_name
                    print self.rhea_360_phone_name.decode('gb2312','ignore')
                    #print self.rhea_360_phone_hot
                    print "*********************"
                    #rhea_brand = 'HTC'
                    phone_name = self.rhea_360_phone_name.decode('gb2312','ignore')
                  #  dbcon.Rhea_db_360_add_phone(self.rhea_db_handle, self.rhea_360_phone_id,self.rhea_360_phone_url,
                  #                              phone_name,self.rhea_360_phone_img, self.rhea_360_phone_price)
                    
            
    def handle_data(self, data):
        if  self.rhea_tag_find:
            #print "Encountered some data  :", data.decode('gb2312')
            if self.rhea_tag_name_find:
                self.rhea_360_phone_name = data


def Rhea_find_uni_block_tag(data,starttag,endtag):
    findstart = data.find(starttag)
    print "uni_block tag find:",findstart
    findend = data[findstart:].find(endtag)
    return data[findstart:findend+findstart+len(endtag)]
    
def Rhea_find_uni_block(data,starttag,endtag):
    findstart = data.find(starttag)
    print "Uni_block find:",findstart
    findend = data[findstart:].find(endtag)
    return data[findstart+len(starttag):findend+findstart]


def Rhea_find_block_tag(data,starttag,endtag):
    
    ltag=[]
    t1 = len(starttag)
    t2 = len(endtag)

    #print t1
    #print t2
    
    i = 0
    n = len(data)
    while(i < n):
    ## find <div
        if data[i] == '<' and data[i:i+t1] == starttag:
            #print "find new tag" , i ,data[i:i+t1] 
            ltag.append(('<',i)) 
        elif data[i] == '<' and data[i:i+t2] == endtag:
            #print "find end tag",i,data[i:i+t2]
            ltag.append(('>',i))
        i = i + 1 
          
    i = 0
    n = len(ltag)
    while(i + 1 < n):
        if ltag[0][0] == '<' and ltag[1][0] == '>':
            break;
        if ltag[i][0] == '<' and ltag[i+1][0] == '>':
            ltag.remove(ltag[i])
            ltag.remove(ltag[i])
            #print i
            #print ltag
            i = 0
            n = len(ltag)
        else:
            i = i + 1
    
    i = ltag[0][1]
    j = ltag[1][1] + t2     
    return data[i:j]  

def Rhea_360buy_page_parser(data):
    pagenum = []
    #pageurllist = []
    
    pattern = Rhea_360buy_html_manager.rhea_360buy_page_pattern
    head = Rhea_360buy_html_manager.rhea_360buy_page_head
    foot = Rhea_360buy_html_manager.rhea_360buy_page_foot
    
    pagelists = re.findall(pattern, data)
    
    if pagelists:
        print "search -->:", pagelists
    else:
        print "No match page lists!!"
    
    for page in pagelists:
        page = re.search('\d{1,}.html$', page)
        page = page.group()
        page = re.match('\d{1,}',page)
        #print page.group()
        pagenum.append(int(page.group()))
    
    pagenum.sort()
    print pagenum
    
    i = min(pagenum)
    j = max(pagenum)
    print "Max pages: = ", j
    
    Rhea_360buy_html_manager.rhea_360buy_page_num = j-i+1 
    
    if j-i+1 != len(pagenum):
        for k in range(i,j+1):
            pageurl = head + str(k) + foot
            print "Find page url=:",pageurl
            Rhea_360buy_html_manager.rhea_360buy_page_lists.append(pageurl)

def Rhea_360buy_process_page(datastr):
    bstart_string = '<div class="pagin fr">'
    bstart_tag = '<div'
    bend_tag = '</div>'

    istart = datastr.find(bstart_string)
    if istart == -1:
        print "Error:Page not find"
        print datastr
        assert 0
    
    data = datastr[istart:]

    sdata = Rhea_find_block_tag(data,bstart_tag,bend_tag)
    print sdata
    
    Rhea_360buy_page_parser(sdata)

    
## process phone    <ul class="list-h">

def Rhea_360buy_process_phone(datastr):
    bstart_string = '<ul class="list-h">'
    bstart_tag = '<ul'
    bend_tag = '</ul>'

    istart = datastr.find(bstart_string)
    print bstart_string
    print "Find block:",istart
    
    if istart == -1:
        print datastr
        assert 0
        
    
    data = datastr[istart:]

    s1 = Rhea_find_block_tag(data,bstart_tag,bend_tag)
    #print s1

    
   # parser = Rhea_360buy_Phone_Parser()
   # parser.rhea_db_handle = dbcon.Rhea_db_connect()
   # parser.feed(s1) 
   # dbcon.Rhea_db_close(parser.rhea_db_handle)       


def Rhea_Get_Html(url):
    request = urllib2.Request(url)
    request.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; rv:10.0.2) Gecko/20100101 Firefox/10.0.2')
    request.add_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
    request.add_header('Accept-Language','zh-cn,en-us;q=0.7,en;q=0.3')
    request.add_header('Accept-Encoding', 'gzip, deflate')
    request.add_header('Connection','keep-alive')
    
    opener = urllib2.build_opener()
    f = opener.open(request)
    
    print f.info()

    compressdata = f.read()
    datatype = f.info().get('Content-Encoding')

    if 'gzip' in datatype:
        datastream = StringIO.StringIO(compressdata)
        gzipper = gzip.GzipFile(fileobj = datastream)
        data = gzipper.read()
        #print data
    f.close()
    return  data


def Rhea_Amazon_process_index(datastr):
    bstart_string = '<div id="leftNavContainer">'
    bstart_tag = '<div'
    bend_tag = '</div>'

    istart = datastr.find(bstart_string)
    if istart == -1:
        print "Error:Page not find"
        #print datastr
        assert 0
    
    data = datastr[istart:]

    sdata = Rhea_find_block_tag(data,bstart_tag,bend_tag)
    #print sdata        
    Rhea_Amazon_html_manager.rhea_Amazon_phone_page = Rhea_Amazon_index_parser(sdata)


def Rhea_Amazon_index_parser(data):
    pstr = '<a href=\"/s/ref=sr_nr_n_0?'

    findstart= data.find(pstr)
    findend= data[findstart:].find('>')

    strurl = data[findstart:findend+findstart+1]

    strlists = strurl.split('\"')
    print strlists[1]
    return strlists[1]


def Rhea_360buy_main_process():
    mainurl = Rhea_360buy_html_manager.rhea_360buy_main_page
    
    mainurlhome = "http://www.amazon.cn/gp/search/ref=sr_hi_1?rh=n%3A664978051&ie=UTF8&qid=1336472328"


    htmldata = Rhea_Get_Html(mainurlhome)

    Rhea_Amazon_process_index(htmldata)
    
    Rhea_Amazon_html_manager.rhea_Amazon_phone_page = "http://www.amazon.cn" + Rhea_Amazon_html_manager.rhea_Amazon_phone_page
    
    print Rhea_Amazon_html_manager.rhea_Amazon_phone_page
    
    htmldata = Rhea_Get_Html(Rhea_Amazon_html_manager.rhea_Amazon_phone_page)
    
    #print htmldata
    
    strdata =  Rhea_find_uni_block_tag(htmldata,'<span class=\"pagnNext\">','</span>')
    
    print strdata
    
    pagnurl =  Rhea_find_uni_block(strdata,'class=\"pagnNext\" href=\"','\">')
    
    print pagnurl
    
    Rhea_Amazon_html_manager.rhea_Amazon_phone_page0 = pagnurl[2:]

    print "phonepage:",Rhea_Amazon_html_manager.rhea_Amazon_phone_page
    print "phonepage0",Rhea_Amazon_html_manager.rhea_Amazon_phone_page0
    
    
    while(Rhea_Amazon_html_manager.rhea_Amazon_has_next_page):
        
    
    
    #print htmldata

    
    return
    
    # process page list html
    Rhea_360buy_process_page(htmldata)

    
    # get phone
    for phonepage in Rhea_360buy_html_manager.rhea_360buy_page_lists:
        print "process-->:",mainurlhome+phonepage
        #fhandle = urllib.urlopen(mainurlhome+phonepage)
        #htmldata = fhandle.read()
        htmldata = Rhea_Get_Html(mainurlhome+phonepage)
        Rhea_360buy_process_phone(htmldata)
        #fhandle.close()
    
    
if __name__ == "__main__":
    Rhea_360buy_main_process()
    
    
    
